Grab lab file using command line:
# Step 1
cd ~/Documents
mkdir lab11
cd lab11
# Step 2
wget https://raw.githubusercontent.com/USCbiostats/PM566/master/website/content/assignment/11-lab.Rmd
And remember to set eval=TRUE
plot_ly() and ggplotly() functionsplot_geo()We will work with COVID data downloaded from the New York Times. The dataset consists of COVID-19 cases and deaths in each US state during the course of the COVID epidemic.
The objective of this lab is to explore relationships between cases, deaths, and population sizes of US states, and plot data to demonstrate this
## data extracted from New York Times state-level data from NYT Github repository
# https://github.com/nytimes/covid-19-data
## state-level population information from us_census_data available on GitHub repository:
# https://github.com/COVID19Tracking/associated-data/tree/master/us_census_data
### FINISH THE CODE HERE ###
# load COVID state-level data from NYT
cv_states <- as.data.frame(data.table::fread("https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-states.csv"))
### FINISH THE CODE HERE ###
# load state population data
state_pops <- as.data.frame(data.table::fread("https://raw.githubusercontent.com/COVID19Tracking/associated-data/master/us_census_data/us_census_2018_population_estimates_states.csv"))
state_pops$abb <- state_pops$state
state_pops$state <- state_pops$state_name
state_pops$state_name <- NULL
### FINISH THE CODE HERE
cv_states <- merge(cv_states, state_pops, by="state")
head, and tail of the datadim(cv_states)
## [1] 51074 9
head(cv_states)
## state date fips cases deaths geo_id population pop_density abb
## 1 Alabama 2021-07-15 1 557578 11439 1 4887871 96.50939 AL
## 2 Alabama 2021-07-28 1 577463 11510 1 4887871 96.50939 AL
## 3 Alabama 2021-08-27 1 681828 12153 1 4887871 96.50939 AL
## 4 Alabama 2021-06-08 1 546540 11220 1 4887871 96.50939 AL
## 5 Alabama 2022-03-17 1 1290692 18998 1 4887871 96.50939 AL
## 6 Alabama 2021-07-30 1 580193 11516 1 4887871 96.50939 AL
tail(cv_states)
## state date fips cases deaths geo_id population pop_density abb
## 51069 Wyoming 2021-11-08 56 105990 1243 56 577737 5.950611 WY
## 51070 Wyoming 2022-11-01 56 178866 1914 56 577737 5.950611 WY
## 51071 Wyoming 2021-08-15 56 68272 793 56 577737 5.950611 WY
## 51072 Wyoming 2022-11-08 56 179366 1917 56 577737 5.950611 WY
## 51073 Wyoming 2022-02-16 56 153663 1689 56 577737 5.950611 WY
## 51074 Wyoming 2022-01-20 56 133495 1601 56 577737 5.950611 WY
str(cv_states)
## 'data.frame': 51074 obs. of 9 variables:
## $ state : chr "Alabama" "Alabama" "Alabama" "Alabama" ...
## $ date : IDate, format: "2021-07-15" "2021-07-28" ...
## $ fips : int 1 1 1 1 1 1 1 1 1 1 ...
## $ cases : int 557578 577463 681828 546540 1290692 580193 551298 1531305 546845 1479605 ...
## $ deaths : int 11439 11510 12153 11220 18998 11516 11358 20533 11249 20048 ...
## $ geo_id : int 1 1 1 1 1 1 1 1 1 1 ...
## $ population : int 4887871 4887871 4887871 4887871 4887871 4887871 4887871 4887871 4887871 4887871 ...
## $ pop_density: num 96.5 96.5 96.5 96.5 96.5 ...
## $ abb : chr "AL" "AL" "AL" "AL" ...
# format the date
cv_states$date <- as.Date(cv_states$date, format="%Y-%m-%d")
# format the state and state abbreviation (abb) variables
state_list <- unique(cv_states$state)
cv_states$state <- factor(cv_states$state, levels = state_list)
abb_list <- unique(cv_states$abb)
cv_states$abb <- factor(cv_states$abb, levels = abb_list)
### FINISH THE CODE HERE
# order the data first by state, second by date
cv_states = cv_states[order(cv_states$state, cv_states$date),]
# Confirm the variables are now correctly formatted
str(cv_states)
## 'data.frame': 51074 obs. of 9 variables:
## $ state : Factor w/ 52 levels "Alabama","Alaska",..: 1 1 1 1 1 1 1 1 1 1 ...
## $ date : Date, format: "2020-03-13" "2020-03-14" ...
## $ fips : int 1 1 1 1 1 1 1 1 1 1 ...
## $ cases : int 6 12 23 29 39 51 78 106 131 157 ...
## $ deaths : int 0 0 0 0 0 0 0 0 0 0 ...
## $ geo_id : int 1 1 1 1 1 1 1 1 1 1 ...
## $ population : int 4887871 4887871 4887871 4887871 4887871 4887871 4887871 4887871 4887871 4887871 ...
## $ pop_density: num 96.5 96.5 96.5 96.5 96.5 ...
## $ abb : Factor w/ 52 levels "AL","AK","AZ",..: 1 1 1 1 1 1 1 1 1 1 ...
head(cv_states)
## state date fips cases deaths geo_id population pop_density abb
## 871 Alabama 2020-03-13 1 6 0 1 4887871 96.50939 AL
## 676 Alabama 2020-03-14 1 12 0 1 4887871 96.50939 AL
## 928 Alabama 2020-03-15 1 23 0 1 4887871 96.50939 AL
## 417 Alabama 2020-03-16 1 29 0 1 4887871 96.50939 AL
## 714 Alabama 2020-03-17 1 39 0 1 4887871 96.50939 AL
## 167 Alabama 2020-03-18 1 51 0 1 4887871 96.50939 AL
tail(cv_states)
## state date fips cases deaths geo_id population pop_density abb
## 50232 Wyoming 2022-11-03 56 178866 1914 56 577737 5.950611 WY
## 50842 Wyoming 2022-11-04 56 178866 1914 56 577737 5.950611 WY
## 50409 Wyoming 2022-11-05 56 178866 1914 56 577737 5.950611 WY
## 50830 Wyoming 2022-11-06 56 178866 1914 56 577737 5.950611 WY
## 50524 Wyoming 2022-11-07 56 178866 1914 56 577737 5.950611 WY
## 51072 Wyoming 2022-11-08 56 179366 1917 56 577737 5.950611 WY
# Inspect the range values for each variable. What is the date range? The range of cases and deaths?
head(cv_states)
## state date fips cases deaths geo_id population pop_density abb
## 871 Alabama 2020-03-13 1 6 0 1 4887871 96.50939 AL
## 676 Alabama 2020-03-14 1 12 0 1 4887871 96.50939 AL
## 928 Alabama 2020-03-15 1 23 0 1 4887871 96.50939 AL
## 417 Alabama 2020-03-16 1 29 0 1 4887871 96.50939 AL
## 714 Alabama 2020-03-17 1 39 0 1 4887871 96.50939 AL
## 167 Alabama 2020-03-18 1 51 0 1 4887871 96.50939 AL
summary(cv_states)
## state date fips cases
## Washington : 1023 Min. :2020-01-21 Min. : 1.00 Min. : 1
## Illinois : 1020 1st Qu.:2020-11-02 1st Qu.:16.00 1st Qu.: 88644
## California : 1019 Median :2021-07-05 Median :29.00 Median : 341318
## Arizona : 1018 Mean :2021-07-05 Mean :29.78 Mean : 811407
## Massachusetts: 1012 3rd Qu.:2022-03-08 3rd Qu.:44.00 3rd Qu.: 954938
## Wisconsin : 1008 Max. :2022-11-08 Max. :72.00 Max. :11390274
## (Other) :44974
## deaths geo_id population pop_density
## Min. : 0 Min. : 1.00 Min. : 577737 Min. : 1.292
## 1st Qu.: 1364 1st Qu.:16.00 1st Qu.: 1805832 1st Qu.: 43.659
## Median : 5104 Median :29.00 Median : 4468402 Median : 107.860
## Mean :11352 Mean :29.78 Mean : 6404128 Mean : 422.942
## 3rd Qu.:14375 3rd Qu.:44.00 3rd Qu.: 7535591 3rd Qu.: 229.511
## Max. :97064 Max. :72.00 Max. :39557045 Max. :11490.120
## NA's :971
## abb
## WA : 1023
## IL : 1020
## CA : 1019
## AZ : 1018
## MA : 1012
## WI : 1008
## (Other):44974
min(cv_states$date)
## [1] "2020-01-21"
max(cv_states$date)
## [1] "2022-11-08"
new_cases and new_deaths and correct outliersAdd variables for new cases, new_cases, and new deaths, new_deaths:
new_cases equal to the difference between cases on date i and date i-1, starting on date i=2Filter to dates after June 1, 2022
Use plotly for EDA: See if there are outliers or values that don’t make sense for new_cases and new_deaths. Which states and which dates have strange values?
Correct outliers: Set negative values for new_cases or new_deaths to 0
Recalculate cases and deaths as cumulative sum of updated new_cases and new_deaths
Get the rolling average of new cases and new deaths to smooth over time
Inspect data again interactively
# Add variables for new_cases and new_deaths:
for (i in 1:length(state_list)) {
cv_subset = subset(cv_states, state == state_list[i])
cv_subset = cv_subset[order(cv_subset$date),]
# add starting level for new cases and deaths
cv_subset$new_cases = cv_subset$cases[1]
cv_subset$new_deaths = cv_subset$deaths[1]
#### FINISH THE CODE HERE ###
for (j in 2:nrow(cv_subset)) {
cv_subset$new_cases[j] = cv_subset$cases[j] - cv_subset$cases[j-1]
cv_subset$new_deaths[j] = cv_subset$deaths[j] - cv_subset$deaths[j-1]
}
# include in main dataset
cv_states$new_cases[cv_states$state==state_list[i]] = cv_subset$new_cases
cv_states$new_deaths[cv_states$state==state_list[i]] = cv_subset$new_deaths
}
cv_states <- cv_states %>% dplyr::filter(date >= "2022-06-01")
### FINISH THE CODE HERE ###
p1<-ggplot(cv_states,
aes( x=date, y=new_cases, color=state )
) + geom_line() + geom_point(size = .5, alpha = 0.5)
ggplotly(p1)
p1<-NULL # to clear from workspace
### FINISH THE CODE HERE ###
p2<-ggplot(cv_states,
aes(x=date, y=new_deaths, color=state )
) + geom_line() + geom_point(size = .5, alpha = 0.5)
ggplotly(p2)
p2<-NULL # to clear from workspace
# set negative new case or death counts to 0
cv_states$new_cases[cv_states$new_cases<0] = 0
cv_states$new_deaths[cv_states$new_deaths<0] = 0
# Recalculate `cases` and `deaths` as cumulative sum of updates `new_cases` and `new_deaths`
for (i in 1:length(state_list)) {
cv_subset = subset(cv_states, state == state_list[i])
# add starting level for new cases and deaths
cv_subset$cases = cv_subset$cases[1]
cv_subset$deaths = cv_subset$deaths[1]
for (j in 2:nrow(cv_subset)) {
cv_subset$cases[j] = cv_subset$new_cases[j] + cv_subset$cases[j-1]
cv_subset$deaths[j] = cv_subset$new_deaths[j] + cv_subset$deaths[j-1]
}
# include in main dataset
cv_states$cases[cv_states$state==state_list[i]] = cv_subset$cases
cv_states$deaths[cv_states$state==state_list[i]] = cv_subset$deaths
}
# Smooth new counts
cv_states$new_cases = zoo::rollmean(cv_states$new_cases, k=7, fill=NA, align='right') %>% round(digits = 0)
cv_states$new_deaths = zoo::rollmean(cv_states$new_deaths, k=7, fill=NA, align='right') %>% round(digits = 0)
# Inspect data again interactively
p2<-ggplot(cv_states, aes(x = date, y = new_deaths, color = state)) + geom_line() + geom_point(size = .5, alpha = 0.5)
ggplotly(p2)
#p2=NULL
# Inspect data again interactively
p2<-ggplot(cv_states, aes(x = date, y = new_deaths, color = state)) + geom_line() + geom_point(size = .5, alpha = 0.5)
ggplotly(p2)
#p2=NULL
Add population-normalized (by 100,000) variables for each variable type (rounded to 1 decimal place). Make sure the variables you calculate are in the correct format (numeric). You can use the following variable names:
per100k = cases per 100,000 populationnewper100k= new cases per 100,000deathsper100k = deaths per 100,000newdeathsper100k = new deaths per 100,000Add a “naive CFR” variable representing deaths / cases on each date for each state
Create a dataframe representing values on the most recent date, cv_states_today, as done in lecture
### FINISH CODE HERE
# add population normalized (by 100,000) counts for each variable
cv_states$per100k = as.numeric(format(round(cv_states$cases/(cv_states$population/100000),1),nsmall=1))
cv_states$newper100k = as.numeric(format(round(cv_states$new_cases/(cv_states$population/100000),1),nsmall=1))
cv_states$deathsper100k = as.numeric(format(round(cv_states$deaths/(cv_states$population/100000),1),nsmall=1))
cv_states$newdeathsper100k = as.numeric(format(round(cv_states$new_deaths/(cv_states$population/100000),1),nsmall=1))
# add a naive_CFR variable = deaths / cases
cv_states = cv_states %>% mutate(naive_CFR = round((deaths*100/cases),2))
# create a `cv_states_today` variable
cv_states_today = subset(cv_states, date==max(cv_states$date))